 #!/usr/bin/env python3
# -*- coding:utf-8 -*-
## author : mike
## date   : 2014.12.4

import os 
import sys
from os.path import realpath, dirname, basename, isdir
from collections import defaultdict
sys.path.append(dirname(dirname(realpath(__file__))))
from xbc3.fileio import FileIO
from xbc3.utilities import *
 
def create_raw_match(inputfile):
    """"""
    counter = defaultdict(lambda:['',0])
    _file = FileIO(inputfile)
    for line in _file:
        line = strip_endl(line)
        (n, url) = line.split(', ', 1)
        url = subhttp(url).rstrip('/')
        n = int(n)
        if url.count('/') >= 2:
            seps = url.split('/', 2)
            seps[2] = '*'
            uw = '/'.join(seps)
            if uw not in counter:
                counter[uw][0] = url
            counter[uw][1] += n
    with FileIO(inputfile+'.2nd', 'w') as fout:
        for k in sorted(counter, key=lambda x:counter[x][1], reverse=True):
            fout.write("%d, %s, %s\n" % (counter[k][1], k, counter[k][0]))
    syserr('raw match file created, %d\n' % len(counter))

create_raw_match('E:\\接收日志文件夹\\result_referer.csv')

    
    